{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "import cv2\n",
    "from sklearn import preprocessing\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.models import Model, load_model, model_from_json\n",
    "from PIL import Image\n",
    "import imagehash\n",
    "import csv\n",
    "from run_experiment import load_params\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = np.array([-0.304255531702266\n",
    ",-0.7589709258986453\n",
    ",-0.08571428571428573\n",
    ",-0.3714285714285715\n",
    ",-0.2571428571428572\n",
    ",-0.7714285714285715])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.4248234572191995"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l.sum() / 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_trainable_layers(model):\n",
    "    trainable_layers = []\n",
    "    i = 0\n",
    "    for layer in model.layers[1:]:\n",
    "        i = i + 1\n",
    "        try:\n",
    "            weights = layer.get_weights()\n",
    "            if (len(weights) == 2 or len(weights) == 1):\n",
    "                trainable_layers.append(model.layers.index(layer))\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    return trainable_layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "origin_model = load_model('data/neural_networks/BIT_DenseNet121.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "layers = get_trainable_layers(origin_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2,\n",
       " 9,\n",
       " 12,\n",
       " 16,\n",
       " 19,\n",
       " 23,\n",
       " 26,\n",
       " 30,\n",
       " 33,\n",
       " 37,\n",
       " 40,\n",
       " 44,\n",
       " 47,\n",
       " 51,\n",
       " 55,\n",
       " 58,\n",
       " 62,\n",
       " 65,\n",
       " 69,\n",
       " 72,\n",
       " 76,\n",
       " 79,\n",
       " 83,\n",
       " 86,\n",
       " 90,\n",
       " 93,\n",
       " 97,\n",
       " 100,\n",
       " 104,\n",
       " 107,\n",
       " 111,\n",
       " 114,\n",
       " 118,\n",
       " 121,\n",
       " 125,\n",
       " 128,\n",
       " 132,\n",
       " 135,\n",
       " 139,\n",
       " 143,\n",
       " 146,\n",
       " 150,\n",
       " 153,\n",
       " 157,\n",
       " 160,\n",
       " 164,\n",
       " 167,\n",
       " 171,\n",
       " 174,\n",
       " 178,\n",
       " 181,\n",
       " 185,\n",
       " 188,\n",
       " 192,\n",
       " 195,\n",
       " 199,\n",
       " 202,\n",
       " 206,\n",
       " 209,\n",
       " 213,\n",
       " 216,\n",
       " 220,\n",
       " 223,\n",
       " 227,\n",
       " 230,\n",
       " 234,\n",
       " 237,\n",
       " 241,\n",
       " 244,\n",
       " 248,\n",
       " 251,\n",
       " 255,\n",
       " 258,\n",
       " 262,\n",
       " 265,\n",
       " 269,\n",
       " 272,\n",
       " 276,\n",
       " 279,\n",
       " 283,\n",
       " 286,\n",
       " 290,\n",
       " 293,\n",
       " 297,\n",
       " 300,\n",
       " 304,\n",
       " 307,\n",
       " 311,\n",
       " 315,\n",
       " 318,\n",
       " 322,\n",
       " 325,\n",
       " 329,\n",
       " 332,\n",
       " 336,\n",
       " 339,\n",
       " 343,\n",
       " 346,\n",
       " 350,\n",
       " 353,\n",
       " 357,\n",
       " 360,\n",
       " 364,\n",
       " 367,\n",
       " 371,\n",
       " 374,\n",
       " 378,\n",
       " 381,\n",
       " 385,\n",
       " 388,\n",
       " 392,\n",
       " 395,\n",
       " 399,\n",
       " 402,\n",
       " 406,\n",
       " 409,\n",
       " 413,\n",
       " 416,\n",
       " 420,\n",
       " 423,\n",
       " 430,\n",
       " 433]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = origin_model.layers[433].get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(weights[0]).shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "213"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.random.choice(layers)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "fault_data = [(64, 402)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n"
     ]
    }
   ],
   "source": [
    "for data in fault_data:\n",
    "    print(data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.choice(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_grad_data(grad_path):\n",
    "    seed_num = 0\n",
    "    num_set = set()\n",
    "    grad_new_inputs = np.load(os.path.join(grad_path, 'new_inputs.npy'))\n",
    "    grad_origin_inputs = np.load(os.path.join(grad_path, 'orgin_inputs.npy'))\n",
    "    grad_new_outputs = np.load(os.path.join(grad_path, 'new_outputs.npy'))\n",
    "    grad_origin_inputs_png = grad_origin_inputs * 255\n",
    "    grad_origin_inputs_png[grad_origin_inputs_png > 255] = 255\n",
    "    grad_origin_inputs_png[grad_origin_inputs_png < 0] = 0\n",
    "    for origin_input in grad_origin_inputs_png:\n",
    "        origin_image = Image.fromarray(np.uint8(origin_input))\n",
    "        origin_image_hash = imagehash.phash(origin_image)\n",
    "        num_set.add(origin_image_hash)\n",
    "    seed_num = len(num_set)\n",
    "#     print(num_set)\n",
    "    print(seed_num)\n",
    "    return grad_new_inputs, grad_origin_inputs, grad_new_outputs, seed_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_deeptest_data(deeptest_path, params, num_class):\n",
    "    deeptest_new_inputs = []\n",
    "    deeptest_origin_inputs = []\n",
    "    deeptest_new_outputs = []\n",
    "    seed_num = 0\n",
    "    num_set = set()\n",
    "    if params == \"GTSRB\":\n",
    "        orgin_path = \"./data/data_set/GTSRB/Final_Training/Images\"\n",
    "        for file in os.listdir(deeptest_path):\n",
    "            lab = int(file[0:5])\n",
    "            num_set.add(file[0:9])\n",
    "            index = file.find('ppm')\n",
    "            origin_file_name = file[:index + 3]\n",
    "            new_file_path = os.path.join(deeptest_path, file)\n",
    "            origin_file_path = orgin_path + '/' + file[0:5] + '/' + origin_file_name\n",
    "            new_img = cv2.imread(new_file_path)\n",
    "            origin_img = cv2.imread(origin_file_path)\n",
    "            new_img = cv2.resize(new_img, (32, 32))\n",
    "            origin_img = cv2.resize(origin_img, (32, 32))\n",
    "            deeptest_new_inputs.append(new_img)\n",
    "            deeptest_origin_inputs.append(origin_img)\n",
    "            deeptest_new_outputs.append(lab)\n",
    "    elif params == \"BIT\":\n",
    "        label_path = \"./data/data_set/BIT_label.csv\"\n",
    "        orgin_path = \"./data/data_set/BIT\"\n",
    "        label_dict = {}\n",
    "        f = csv.reader(open(label_path, 'r', encoding='utf-8'))\n",
    "        for lists in f:\n",
    "            label_dict[lists[1]] = lists[2]\n",
    "\n",
    "        for file in os.listdir(deeptest_path):\n",
    "            origin_file_name = file[:19]\n",
    "            num_set.add(file[0:19])\n",
    "            new_file_path = os.path.join(deeptest_path, file)\n",
    "            origin_file_path = orgin_path + '/' + origin_file_name\n",
    "            new_img = cv2.imread(new_file_path)\n",
    "            origin_img = cv2.imread(origin_file_path)\n",
    "            new_img = cv2.resize(new_img, (32, 32))\n",
    "            origin_img = cv2.resize(origin_img, (32, 32))\n",
    "            deeptest_new_inputs.append(new_img)\n",
    "            deeptest_origin_inputs.append(origin_img)\n",
    "            deeptest_new_outputs.append(int(label_dict[origin_file_name]))\n",
    "    elif params == \"Car\":\n",
    "        label_path = \"./data/data_set/Car_label.csv\"\n",
    "        orgin_path = \"./data/data_set/car_ims\"\n",
    "        f = csv.reader(open(label_path, 'r', encoding='utf-8'))\n",
    "        label_dict = {}\n",
    "        for lists in f:\n",
    "            label_dict[lists[1]] = lists[2]\n",
    "\n",
    "        for file in os.listdir(deeptest_path):\n",
    "            origin_file_name = file[:10]\n",
    "            num_set.add(file[0:10])\n",
    "            new_file_path = os.path.join(deeptest_path, file)\n",
    "            origin_file_path = orgin_path + '/' + origin_file_name\n",
    "            new_img = cv2.imread(new_file_path)\n",
    "            origin_img = cv2.imread(origin_file_path)\n",
    "            new_img = cv2.resize(new_img, (32, 32))\n",
    "            origin_img = cv2.resize(origin_img, (32, 32))\n",
    "            deeptest_new_inputs.append(new_img)\n",
    "            deeptest_origin_inputs.append(origin_img)\n",
    "            deeptest_new_outputs.append(int(label_dict['car_ims/'+origin_file_name]))\n",
    "\n",
    "    deeptest_new_inputs = np.array(deeptest_new_inputs)\n",
    "    deeptest_origin_inputs = np.array(deeptest_origin_inputs)\n",
    "    deeptest_new_outputs = np.array(deeptest_new_outputs)\n",
    "    lb = preprocessing.LabelBinarizer().fit(np.array(range(num_class)))  # 对标签进行ont_hot编码\n",
    "    deeptest_new_outputs = lb.transform(deeptest_new_outputs)  # 因为是多分类任务，必须进行编码处理\n",
    "    seed_num = len(num_set)\n",
    "    print(seed_num)\n",
    "    return deeptest_new_inputs, deeptest_origin_inputs, deeptest_new_outputs, seed_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2456\n",
      "{'vehicle_0009324.jpg', 'vehicle_0009014.jpg'}\n",
      "2\n",
      "None Fidelity is begining:\n",
      "grad_len: 3072\n",
      "deeptest_len: 1998\n"
     ]
    }
   ],
   "source": [
    "model = \"ResNet50\"\n",
    "data = \"BIT\"\n",
    "model_path = \"./data/neural_networks\"\n",
    "# model = params.model\n",
    "grad_path = \"D:\\Projects\\PycharmProjects\\Grad_/output/grad/\" + model + '/' + data + '/' + \"generated_samples\"\n",
    "# deepsmartfuzz_path = params.deepsmartfuzz_path\n",
    "deeptest_path = \"D:\\Projects\\PycharmProjects\\Grad_/output/deeptest/\" + model + '/' + data + '/' + \"new\"\n",
    "# deeptest_path = params.deeptest_path\n",
    "# BIT_path = params.BIT_path\n",
    "# model_path = params.model_path\n",
    "num_class = 6\n",
    "grad_new_inputs, grad_origin_inputs, grad_new_outputs, seed_num = get_grad_data(grad_path)\n",
    "deeptest_new_inputs, deeptest_origin_inputs, deeptest_new_outputs, seed_num = get_deeptest_data(deeptest_path, \"BIT\", num_class)\n",
    "print(\"None Fidelity is begining:\")\n",
    "# deepsmartfuzz_new_inputs_temp = deepsmartfuzz_new_inputs / 255.\n",
    "deeptest_new_inputs = deeptest_new_inputs / 255.\n",
    "print(\"grad_len: \" + str(len(grad_new_inputs)))\n",
    "print(\"deeptest_len: \" + str(len(deeptest_new_inputs)))\n",
    "model_name = data + '_' + model + '.h5'\n",
    "origin_model = load_model(os.path.join(model_path, model_name))\n",
    "# origin_model = load_model(model_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = argparse.ArgumentParser(description=\"Experiments Script For evaluate\")\n",
    "parser.add_argument(\"--params_set\", nargs='*', type=str, default=[\"Car\", \"VGG16\"],\n",
    "                    help=\"see params folder\")\n",
    "parser.add_argument(\"--model\", type=str, default=\"VGG16\", choices=[\"INCEPTION_NET\", \"VGG16\", \"ResNet50\", \"DenseNet121\"])\n",
    "parser.add_argument(\"--data\", type=str, default=\"Car\", choices=[\"BKK100\", \"GTSRB\", \"BIT\", \"Car\"])\n",
    "parser.add_argument(\"--approach\", type=str, default=\"tarantula\")\n",
    "parser.add_argument(\"--model_path\", type=str, default=\"./data/neural_networks\")\n",
    "parser.add_argument(\"--deepfault_path\", type=str, default=\"./output/deepfault\")\n",
    "parser.add_argument(\"--mutant_path\", type=str, default=\"./output/mutant\")\n",
    "# parser.add_argument(\"--grad_path\", type=str, default=\"data/result/deepsmartfuzz/3072-15h\")\n",
    "# parser.add_argument(\"--deeptest_path\", type=str, default=\"data/result/deeptest/100-0.2-36h/new\")\n",
    "# parser.add_argument(\"--model_path\", type=str, default=\"./data/neural_networks/BIT_RESNET50_model.h5\")\n",
    "# parser.add_argument(\"--BIT_path\", type=str, default=\"./dataset/BITVehicle_Dataset\")\n",
    "params = parser.parse_args()\n",
    "start_time = time.time()\n",
    "run_evaluate(params)\n",
    "time_passed_min = (time.time() - start_time) / 60\n",
    "print(\"time passed (minutes): %g\" % time_passed_min)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "grad",
   "language": "python",
   "name": "grad"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
